import pandas as pd
from html2text import html2text
import re

# zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/poc_1.7w+_data/2024-03-29_市热线-知识库点点通信息_17603.csv'
# sanya_zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/三亚知识库数据/补充-三亚问答数据0117.csv'
# sanya_zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/三亚知识库数据/补充2-三亚1咨询归档随机抽14800(1).csv'
sanya_zsk_data_path = r'/workspace/qanything_local/qanything_kernel/row_data/三亚知识库数据/补充2-三亚咨询类清单（2023.10-2023.11）.csv'

# df = pd.read_csv(zsk_data_path)
df = pd.read_csv(sanya_zsk_data_path, encoding='utf8')  # ['KL_ID', 'KL_CONTENT_KEYWORD', 'KL_CONTENT_TITLE', 'KL_CONTENT', 'KL_CONTENT_TYPE']

# 去除 '内容' 列中的缺失值
# df.dropna(subset=['内容'], inplace=True)
print(len(df))
df.dropna(subset=['最终处理描述'], inplace=True)

import os
import csv
from uuid import uuid4

# db_dir = r'/workspace/qanything_local/QANY_LOCAL_DB/sanya_zsk-2024-04-16_600'
# db_dir = r'/workspace/qanything_local/QANY_LOCAL_DB/sanya_zsk-2024-04-18_14K'
db_dir = r'/workspace/qanything_local/QANY_LOCAL_DB/sanya_zsk-2024-04-18_14+43K'

def truncate_title(title, max_length=25):
    """截断标题到指定长度，并移除不安全的文件名字符"""
    # 移除不安全的文件名字符
    safe_title = re.sub(r'[\\/*?:"<>|]', '_', title)
    # 截断标题到最大长度
    return safe_title[:max_length].strip()

def write_to_csv(zsk_file_id, title, key_words, content):
    # 确保文件名是安全的，不会与文件系统的路径分隔符冲突
    title = truncate_title(title)

    # 根据 title 生成文件名
    if title:
        file_name = f"{title}.csv"
    else:
        file_name = f"{uuid4()}.csv"
    file_path = os.path.join(db_dir, file_name)

    # 如果文件已存在，则添加唯一标识符
    if os.path.exists(file_path):
        file_name = f"{title}_{uuid4()}.csv"
        file_path = os.path.join(db_dir, file_name)

    # 创建 CSV 文件并写入数据
    try:
        with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
            writer = csv.writer(csvfile)
            # 写入标题行
            writer.writerow(['zsk_file_id', 'title', 'keywords', 'content'])
            # 写入内容行
            writer.writerow([zsk_file_id, title, key_words, content])
    except Exception as e:
        print(f'写入文件时出现错误：{e}')
        # 如果写入失败，尝试创建一个新的文件名
        file_name = f"{uuid.uuid4()}.csv"
        file_path = os.path.join(db_dir, file_name)
        try:
            with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
                writer = csv.writer(csvfile)
                writer.writerow(['zsk_file_id', 'title', 'keywords', 'content'])
                writer.writerow([zsk_file_id, title, key_words, content])
        except Exception as e:
            print(f'创建新文件时出现错误：{e}')
        else:
            pass  

print('过滤后文件数量：', len(df))
for i in range(len(df)):
# for i in range(2):  # test
    try:
        zsk_file_id = ''
        title = df.iloc[i]['工单标题']
        question = df.iloc[i]['工单标题']
        content = html2text(df.iloc[i]['最终处理描述'])

        # if pd.isna(df.iloc[i]['关键字']):
        #     key_words = ''
        #     kb_content ='问题：' + question + '\n' + '内容：\n' + content
        # else:
        #     key_words = df.iloc[i]['关键字']
        #     kb_content = '内容关键字：' + key_words + '\n' + '问题：' + question + '\n' + '内容：\n' + content

        key_words = ''
        kb_content ='工单标题：\n' + question + '\n' + '工单内容：\n' + content

        kb_content = re.sub(r'\n+', '\n', kb_content)
        kb_content = kb_content.rstrip('\n')
        write_to_csv(zsk_file_id, title, key_words, kb_content)
    
    except Exception as e:
        print(f'{i+1}文件存在问题,请检查')
        continue

# 查看文件夹下文件数量
# ls -l | grep ^- | wc -l